import os
import subprocess
import sys
import time
from datetime import datetime

import redis
import requests

# Redis 配置
sRedisHost = "r-j6cmt5ayr2nq1m6zpp.redis.rds.aliyuncs.com"
sRedisPWD = "r-j6cmt5ayr2nq1m6zpp:qpalmz1209@L"
sRedisDB = 0

# 出图机器ID
aMachineID = [
    "ws-1",
    "ws-2",
    "ws-3",
    "ws-4"
]

# 重启阈值 - 队列长度
iLenRestartThreshold = 50
# 重启阈值 - 间隔时间
iTimeRestartThreshold = 600

# 记录重启时间
aRestartTime = {
    "ws-1": 1,
    "ws-2": 0,
    "ws-3": 0,
    "ws-4": 0
}

# redis 链接
# try:
#     oRedis = redis.StrictRedis(host=sRedisHost, port=6379, db=sRedisDB, password=sRedisPWD, socket_connect_timeout=10)
#     print("connected")
#     oRedis.close()
#     print("closed")
# except Exception as E:
#     print("Unable to reset cached search results: {}".format(E))


# 写日志
def writeLog(sStr):
    sDate = datetime.now().strftime('%Y-%m-%d')
    sDateTime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    sPath = "../logs/monitor_" + sDate + ".txt"
    oFile = open(sPath, 'a+')
    oFile.write('[' + sDateTime + ']' + sStr + '\n')
    oFile.close()


# 获取时间戳
def getTimestamp():
    oNow = datetime.now()
    iTimestamp = datetime.timestamp(oNow)
    return int(str(iTimestamp).split('.')[0])


# 重启指定关键字进程
def restartProcessByKey(sKey):
    # 取进程号
    sCommand = "ps -ef | grep " + sKey + " |  grep -v grep | awk '{print $2}'"
    sOutput = os.popen(sCommand).read()
    aProcessNu = sOutput.splitlines()
    for sProcessNu in aProcessNu:
        sCommand = "kill -9 " + sProcessNu
        os.system(sCommand)
        writeLog('kill ' + sKey + ' ' + sProcessNu)


# 处理重启
def handleRestart(sMachineID):
    iTimestamp = getTimestamp()
    iInterval = iTimestamp - aRestartTime[sMachineID]
    # print('handleRestart :' + str(iInterval) + ' > ' + str(iTimeRestartThreshold))
    if iInterval > iTimeRestartThreshold:
        restartProcessByKey(sMachineID)


def monitor():
    # redis 链接
    try:
        oRedis = redis.StrictRedis(host=sRedisHost, port=6379, db=sRedisDB, password=sRedisPWD,socket_connect_timeout=10)
        print("connected")
    except Exception as E:
        print("Unable to reset cached search results: {}".format(E))

    for sMachineID in aMachineID:
        sQueueKey = "painter:{}:queue".format(sMachineID)
        iLen = oRedis.llen(sQueueKey)
        # print(sMachineID + ' queue :' + str(iLen) + ' > ' + str(iLenRestartThreshold))
        writeLog(sMachineID + ' queue len:' + iLen)
        if iLen > iLenRestartThreshold:
            handleRestart(sMachineID)
    oRedis.close()
    print("closed")

while True:
    monitor()
    time.sleep(60)
